{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spider 'liepinSpider' already exists in module:\n",
      "  liepin.spiders.liepinSpider\n"
     ]
    }
   ],
   "source": [
    "! scrapy genspider liepinSpider \"https://www.liepin.com/zhaopin/?industries=&subIndustry=&dqs=050090&salary=&jobKind=&pubTime=&compkind=&compscale=&searchType=1&isAnalysis=&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7E-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-07-19 14:38:31 [scrapy.utils.log] INFO: Scrapy 1.6.0 started (bot: liepin)\n",
      "2020-07-19 14:38:31 [scrapy.utils.log] INFO: Versions: lxml 4.5.0.0, libxml2 2.9.9, cssselect 1.1.0, parsel 1.5.2, w3lib 1.21.0, Twisted 20.3.0, Python 3.7.3 (default, Apr 24 2019, 15:29:51) [MSC v.1915 64 bit (AMD64)], pyOpenSSL 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020), cryptography 2.8, Platform Windows-10-10.0.17134-SP0\n",
      "2020-07-19 14:38:31 [scrapy.crawler] INFO: Overridden settings: {'BOT_NAME': 'liepin', 'NEWSPIDER_MODULE': 'liepin.spiders', 'SPIDER_MODULES': ['liepin.spiders']}\n",
      "2020-07-19 14:38:31 [scrapy.extensions.telnet] INFO: Telnet Password: fdefec6784721de0\n",
      "2020-07-19 14:38:31 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2020-07-19 14:38:34 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2020-07-19 14:38:34 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2020-07-19 14:38:34 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "['liepin.pipelines.LiepinPipeline']\n",
      "2020-07-19 14:38:34 [scrapy.core.engine] INFO: Spider opened\n",
      "2020-07-19 14:38:34 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2020-07-19 14:38:34 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=1> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=2> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=0> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=4> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=3> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=6> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=5> (referer: None)\n",
      "2020-07-19 14:38:35 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=7> (referer: None)\n",
      "2020-07-19 14:38:36 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['某互联网保险公司',\n",
      "                      '国内照明行业领军品牌',\n",
      "                      '外资某知名日用品公司',\n",
      "                      '高端离子源技术研发及整机装备制造公司',\n",
      "                      '地产百强',\n",
      "                      '深圳某大数据征信科技公司',\n",
      "                      '某企业',\n",
      "                      '某阿里系跨境电商平台开发及运营公司',\n",
      "                      '某阿里系跨境电商平台开发及运营公司',\n",
      "                      '某信息科技公司',\n",
      "                      '广东医疗器械集团',\n",
      "                      '国内知名互联网上市公司',\n",
      "                      '国内知名互联网上市公司',\n",
      "                      '广东医疗器械集团',\n",
      "                      '某大型港资上市公司',\n",
      "                      '深圳某跨境电商公司',\n",
      "                      '某知名公司',\n",
      "                      '广东省某焊接激光设备企业',\n",
      "                      '某公司',\n",
      "                      '地产百强',\n",
      "                      '某TOP级央企科技集团',\n",
      "                      '某TOP级央企科技集团',\n",
      "                      '某D轮公司',\n",
      "                      '某互联网音乐公司',\n",
      "                      '某互联网音乐公司',\n",
      "                      '某互联网音乐公司',\n",
      "                      '香港某上市公司',\n",
      "                      '深圳坂田某知名精品跨境电商公司',\n",
      "                      '某通讯公司',\n",
      "                      '坂田地铁站某精品跨境电商公司',\n",
      "                      '某互联网巨头',\n",
      "                      '某上市集团公司',\n",
      "                      '某动漫平台',\n",
      "                      '某TOP级央企科技集团',\n",
      "                      '某TOP级央企科技集团',\n",
      "                      '某TOP级央企科技集团',\n",
      "                      '某网络技术（北京）有限公司',\n",
      "                      '某网络技术（北京）有限公司',\n",
      "                      '某网络技术（北京）有限公司',\n",
      "                      '深圳坂田某知名精品跨境电商公司'],\n",
      " 'job_company_url': [],\n",
      " 'job_url': ['/a/21063237.shtml',\n",
      "             '/a/21044941.shtml',\n",
      "             '/a/21030561.shtml',\n",
      "             '/a/20981223.shtml',\n",
      "             '/a/20960005.shtml',\n",
      "             '/a/20955717.shtml',\n",
      "             '/a/20953291.shtml',\n",
      "             '/a/20943635.shtml',\n",
      "             '/a/20943633.shtml',\n",
      "             '/a/20932137.shtml',\n",
      "             '/a/20903879.shtml',\n",
      "             '/a/20894201.shtml',\n",
      "             '/a/20894029.shtml',\n",
      "             '/a/20890903.shtml',\n",
      "             '/a/20885077.shtml',\n",
      "             '/a/20872037.shtml',\n",
      "             '/a/20809881.shtml',\n",
      "             '/a/20801681.shtml',\n",
      "             '/a/20790793.shtml',\n",
      "             '/a/20741097.shtml',\n",
      "             '/a/20616023.shtml',\n",
      "             '/a/20615711.shtml',\n",
      "             '/a/20608527.shtml',\n",
      "             '/a/20578271.shtml',\n",
      "             '/a/20578265.shtml',\n",
      "             '/a/20578257.shtml',\n",
      "             '/a/20464637.shtml',\n",
      "             '/a/20451815.shtml',\n",
      "             '/a/20423519.shtml',\n",
      "             '/a/20415289.shtml',\n",
      "             '/a/20343091.shtml',\n",
      "             '/a/20307503.shtml',\n",
      "             '/a/20217237.shtml',\n",
      "             '/a/20149947.shtml',\n",
      "             '/a/20146919.shtml',\n",
      "             '/a/20146905.shtml',\n",
      "             '/a/20075307.shtml',\n",
      "             '/a/20075297.shtml',\n",
      "             '/a/20075275.shtml',\n",
      "             '/a/20023213.shtml'],\n",
      " 'job_xinshui': ['20-30k·15薪',\n",
      "                 '20-30k·14薪',\n",
      "                 '15-20k·15薪',\n",
      "                 '16-20k·13薪',\n",
      "                 '25-50k·12薪',\n",
      "                 '40-50k·15薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '25-45k·12薪',\n",
      "                 '30-40k·12薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '25-50k·16薪',\n",
      "                 '30-55k·16薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '15-20k·13薪',\n",
      "                 '40-51k·12薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '22-35k·12薪',\n",
      "                 '30-60k·16薪',\n",
      "                 '30-40k·15薪',\n",
      "                 '30-50k·14薪',\n",
      "                 '30-60k·15薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '50-65k·12薪',\n",
      "                 '18-28k·13薪',\n",
      "                 '35-42k·12薪',\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                 '25-35k·13薪',\n",
      "                 '25-50k·16薪',\n",
      "                 '180-200k·12薪',\n",
      "                 '25-35k·13薪',\n",
      "                 '30-40k·15薪',\n",
      "                 '35-50k·14薪',\n",
      "                 '35-50k·15薪',\n",
      "                 '35-50k·15薪',\n",
      "                 '35-50k·15薪',\n",
      "                 '35-60k·15薪',\n",
      "                 '20-32k·13薪'],\n",
      " 'job_zhicheng': ['产品经理',\n",
      "                  '产品（灯饰照明）经理/高级经理',\n",
      "                  '产品工业设计经理',\n",
      "                  '销售部产品经理（真空镀膜设备 离子源）售前技术支持',\n",
      "                  '产品经理',\n",
      "                  '产品高级经理/总监',\n",
      "                  '产品经理',\n",
      "                  '高级产品经理（APOS）',\n",
      "                  '高级产品经理(APOLLO)',\n",
      "                  '互联网营销业务专家/高级产品经理',\n",
      "                  '微分销母婴服装社交电商运营产品商品经理S2B2C社交消费者运营',\n",
      "                  '物联网开发平台高级产品拓展经理',\n",
      "                  '物联网产品高级运营经理',\n",
      "                  '社交电商运营产品经理S2B2C社交消费者运营',\n",
      "                  '产品经理',\n",
      "                  '跨境电商产品开发经理（选品）',\n",
      "                  '地图数据产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（互联网或云平台经验优先）',\n",
      "                  '产品经理/专家',\n",
      "                  '租赁产品经理-房地产业务',\n",
      "                  'BPM产品经理（地产业务流程）',\n",
      "                  '产品经理/高级产品经理',\n",
      "                  '曲库产品经理',\n",
      "                  '直播产品经理',\n",
      "                  'K歌产品经理',\n",
      "                  '金融产品经理',\n",
      "                  '产品经理（键鼠）',\n",
      "                  '产品经理（手表行业）',\n",
      "                  '产品经理（小家电）',\n",
      "                  '行业营销高级经理（支付产品）',\n",
      "                  '大数据产品经理',\n",
      "                  '产品经理',\n",
      "                  '中台产品经理',\n",
      "                  '产业新城产品经理',\n",
      "                  '酒店产品经理',\n",
      "                  '高级中后台产品经理',\n",
      "                  '高级产品经理-商户方向',\n",
      "                  '高级产品策划经理',\n",
      "                  '宠物用品产品经理'],\n",
      " 'liepin_jingyan': ['3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '10年以上',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '10年以上',\n",
      "                    '经验不限',\n",
      "                    '1-3年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '经验不限'],\n",
      " 'liepin_xueli': ['统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '学历不限',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '硕士及以上',\n",
      "                  '统招本科']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=9> (referer: None)\n",
      "2020-07-19 14:38:36 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=8> (referer: None)\n",
      "2020-07-19 14:38:37 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['知名企业',\n",
      "                      '某TOP国有公募基金公司',\n",
      "                      '深圳坂田某精品电商公司',\n",
      "                      '深圳某公司',\n",
      "                      '腾讯',\n",
      "                      '腾讯',\n",
      "                      '腾讯',\n",
      "                      '深圳华文网教育科技有限公司',\n",
      "                      '深圳市乐黛品牌管理有限公司',\n",
      "                      'vivo',\n",
      "                      '前海手绘',\n",
      "                      '深圳市昇利扬科技有限公司',\n",
      "                      '深圳市昇利扬科技有限公司',\n",
      "                      '景瀚科技(深圳)有限公司',\n",
      "                      '丝路视觉',\n",
      "                      '丝路视觉',\n",
      "                      '深圳中电国际信息科技有限公司',\n",
      "                      '信必优',\n",
      "                      '深圳九章数据科技有限公司',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '深圳市极摩科技有限公司',\n",
      "                      '深圳市摩乐吉科技有限公司',\n",
      "                      '深圳市摩乐吉科技有限公司',\n",
      "                      '深圳市傲雷电商科技股份有限公司',\n",
      "                      '赛维网络',\n",
      "                      '半岛医疗',\n",
      "                      '鹏元征信',\n",
      "                      '深圳市泰普达科技有限公司',\n",
      "                      '赛维网络',\n",
      "                      '电目科技',\n",
      "                      '深圳市前海云途物流有限公司',\n",
      "                      '腾讯',\n",
      "                      '深圳市耐拓能源科技有限公司',\n",
      "                      '深圳警翼智能科技股份有限公司',\n",
      "                      '赛维网络',\n",
      "                      '土巴兔',\n",
      "                      '来电科技',\n",
      "                      '深圳市傲雷电商科技股份有限公司',\n",
      "                      '杰诺智能科技有限公司',\n",
      "                      '深圳市财盈通科技有限公司'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/12269863/',\n",
      "                     'https://www.liepin.com/company/12180075/',\n",
      "                     'https://www.liepin.com/company/4787049/',\n",
      "                     'https://www.liepin.com/company/8629990/',\n",
      "                     'https://www.liepin.com/company/8162234/',\n",
      "                     'https://www.liepin.com/company/8162234/',\n",
      "                     'https://www.liepin.com/company/8199066/',\n",
      "                     'https://www.liepin.com/company/9143699/',\n",
      "                     'https://www.liepin.com/company/9143699/',\n",
      "                     'https://www.liepin.com/company/8523053/',\n",
      "                     'https://www.liepin.com/company/7879896/',\n",
      "                     'https://www.liepin.com/company/10188067/',\n",
      "                     'https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/10233137/',\n",
      "                     'https://www.liepin.com/company/9997741/',\n",
      "                     'https://www.liepin.com/company/9997741/',\n",
      "                     'https://www.liepin.com/company/10282081/',\n",
      "                     'https://www.liepin.com/company/7983265/',\n",
      "                     'https://www.liepin.com/company/8849807/',\n",
      "                     'https://www.liepin.com/company/8088241/',\n",
      "                     'https://www.liepin.com/company/12210023/',\n",
      "                     'https://www.liepin.com/company/7983265/',\n",
      "                     'https://www.liepin.com/company/12252503/',\n",
      "                     'https://www.liepin.com/company/10032247/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/7988838/',\n",
      "                     'https://www.liepin.com/company/8497137/',\n",
      "                     'https://www.liepin.com/company/7983265/',\n",
      "                     'https://www.liepin.com/company/5305829/',\n",
      "                     'https://www.liepin.com/company/8711037/',\n",
      "                     'https://www.liepin.com/company/10282081/',\n",
      "                     'https://www.liepin.com/company/12288013/',\n",
      "                     'https://www.liepin.com/company/10040913/'],\n",
      " 'job_url': ['/a/19855079.shtml',\n",
      "             '/a/18497589.shtml',\n",
      "             '/a/17547951.shtml',\n",
      "             '/a/21038449.shtml',\n",
      "             'https://www.liepin.com/job/1926051717.shtml',\n",
      "             'https://www.liepin.com/job/1927740561.shtml',\n",
      "             'https://www.liepin.com/job/1927359997.shtml',\n",
      "             'https://www.liepin.com/job/1930036041.shtml',\n",
      "             'https://www.liepin.com/job/1930035363.shtml',\n",
      "             'https://www.liepin.com/job/1930034961.shtml',\n",
      "             'https://www.liepin.com/job/1930032251.shtml',\n",
      "             'https://www.liepin.com/job/1930030241.shtml',\n",
      "             'https://www.liepin.com/job/1930030239.shtml',\n",
      "             'https://www.liepin.com/job/1930029869.shtml',\n",
      "             'https://www.liepin.com/job/1930029807.shtml',\n",
      "             'https://www.liepin.com/job/1930029799.shtml',\n",
      "             'https://www.liepin.com/job/1930029445.shtml',\n",
      "             'https://www.liepin.com/job/1930027891.shtml',\n",
      "             'https://www.liepin.com/job/1930003265.shtml',\n",
      "             'https://www.liepin.com/job/1929976307.shtml',\n",
      "             'https://www.liepin.com/job/1929952195.shtml',\n",
      "             'https://www.liepin.com/job/1929911713.shtml',\n",
      "             'https://www.liepin.com/job/1929911585.shtml',\n",
      "             'https://www.liepin.com/job/1929890293.shtml',\n",
      "             'https://www.liepin.com/job/1929883511.shtml',\n",
      "             'https://www.liepin.com/job/1929883171.shtml',\n",
      "             'https://www.liepin.com/job/1929872487.shtml',\n",
      "             'https://www.liepin.com/job/1929791123.shtml',\n",
      "             'https://www.liepin.com/job/1929747755.shtml',\n",
      "             'https://www.liepin.com/job/1929737549.shtml',\n",
      "             'https://www.liepin.com/job/1929689785.shtml',\n",
      "             'https://www.liepin.com/job/1929636519.shtml',\n",
      "             'https://www.liepin.com/job/1929636265.shtml',\n",
      "             'https://www.liepin.com/job/1929587417.shtml',\n",
      "             'https://www.liepin.com/job/1929569119.shtml',\n",
      "             'https://www.liepin.com/job/1929531661.shtml',\n",
      "             'https://www.liepin.com/job/1929502027.shtml',\n",
      "             'https://www.liepin.com/job/1929443999.shtml',\n",
      "             'https://www.liepin.com/job/1929443383.shtml',\n",
      "             'https://www.liepin.com/job/1929440309.shtml'],\n",
      " 'job_xinshui': ['30-60k·16薪',\n",
      "                 '30-40k·24薪',\n",
      "                 '15-28k·13薪',\n",
      "                 '20-30k·14薪',\n",
      "                 '面议',\n",
      "                 '面议',\n",
      "                 '面议',\n",
      "                 '7-10k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '25-50k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '20-25k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '10-30k·12薪',\n",
      "                 '14-18k·12薪',\n",
      "                 '12-20k·12薪',\n",
      "                 '10-18k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '12-15k·12薪',\n",
      "                 '12-17k·12薪',\n",
      "                 '10-15k·14薪',\n",
      "                 '10-20k·12薪',\n",
      "                 '10-20k·12薪',\n",
      "                 '25-45k·12薪',\n",
      "                 '8-12k·12薪',\n",
      "                 '10-20k·13薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '10-18k·12薪',\n",
      "                 '面议',\n",
      "                 '15-35k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '13-18k·12薪',\n",
      "                 '35-45k·12薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '10-15k·14薪',\n",
      "                 '8-23k·12薪',\n",
      "                 '15-30k·13薪'],\n",
      " 'job_zhicheng': ['产品经理',\n",
      "                  '高级产品经理',\n",
      "                  'TWS产品经理',\n",
      "                  '设备PM',\n",
      "                  '18402-韩语PM',\n",
      "                  '37088-游戏海外PM（深圳）',\n",
      "                  '26563-WeGame游戏英语海外PM(深圳)',\n",
      "                  '课程产品经理',\n",
      "                  '（美容仪）产品经理',\n",
      "                  '应用分发产品经理（广告方向）(008005)',\n",
      "                  '移动端产品经理',\n",
      "                  '产品经理（电子类）',\n",
      "                  '产品经理（材料类）',\n",
      "                  'B端产品经理',\n",
      "                  'VR产品经理（K12教育）',\n",
      "                  '高级产品经理（大数据可视化）',\n",
      "                  '产品经理 PM FPGA Lattice',\n",
      "                  '产品经理（中高级）',\n",
      "                  '大数据产品经理',\n",
      "                  '产品经理（POS业务）(J13646)',\n",
      "                  '产品经理（家电产品）',\n",
      "                  '产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '增长产品经理',\n",
      "                  'ERP产品经理',\n",
      "                  '皮肤科医生/医疗美容/产品经理',\n",
      "                  '产品负责人/高级产品经理',\n",
      "                  '产品开发专员/经理',\n",
      "                  '产品经理（工具/汽摩/小家电方向）',\n",
      "                  '产品经理（硬件）',\n",
      "                  '跨境物流产品经理',\n",
      "                  '22989-基础产品运营经理',\n",
      "                  '急聘电商产品经理（3C类目）',\n",
      "                  '软件产品经理',\n",
      "                  '产品经理（跑步机方向）',\n",
      "                  '高级产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '互联网产品经理',\n",
      "                  '产品总监/经理(吸尘器及清洁设备)',\n",
      "                  '产品经理（浏览器生态方向）'],\n",
      " 'liepin_jingyan': ['3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '一年以下',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '3-5年'],\n",
      " 'liepin_xueli': ['学历不限',\n",
      "                  '硕士及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '学历不限',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:38 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['深圳市聚创意创新科技有限公司',\n",
      "                      '深圳市同行者科技有限公司',\n",
      "                      '递四方',\n",
      "                      '深圳哈希坊科技有限公司',\n",
      "                      '深圳浅桥文化传播有限公司',\n",
      "                      '深圳九章数据科技有限公司',\n",
      "                      '兔展-RabbitPre',\n",
      "                      '深圳市海果达科技有限公司',\n",
      "                      '蓝凌软件',\n",
      "                      'vivo',\n",
      "                      '深圳莱福德科技股份有限公司',\n",
      "                      '深圳乐木骆科技有限公司',\n",
      "                      '智令互动(深圳)科技有限公司',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '豪恩科技',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '理邦仪器',\n",
      "                      '百旺互联网金融服务(深圳)有限公司',\n",
      "                      '深圳乐木骆科技有限公司',\n",
      "                      '深圳乐木骆科技有限公司',\n",
      "                      '深圳乐木骆科技有限公司',\n",
      "                      '腾讯',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '深圳市几米物联有限公司',\n",
      "                      '深圳市九方通逊电商物流有限公司',\n",
      "                      '腾讯',\n",
      "                      '深圳市客户通科技有限公司',\n",
      "                      '51互联网公司',\n",
      "                      '国内500强企业',\n",
      "                      '大数据征信公司（持牌照）',\n",
      "                      '某知名上市企业',\n",
      "                      '湖北三赢兴电子科技有限公司',\n",
      "                      '某知名在线英语教育企业',\n",
      "                      '某公司',\n",
      "                      '大型上市集团公司',\n",
      "                      '国内知名互联网企业',\n",
      "                      '国内知名互联网企业',\n",
      "                      '深圳某网络科技有限公司'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/8740703/',\n",
      "                     'https://www.liepin.com/company/9582294/',\n",
      "                     'https://www.liepin.com/company/3645457/',\n",
      "                     'https://www.liepin.com/company/12315857/',\n",
      "                     'https://www.liepin.com/company/12315749/',\n",
      "                     'https://www.liepin.com/company/10188067/',\n",
      "                     'https://www.liepin.com/company/8161746/',\n",
      "                     'https://www.liepin.com/company/9215423/',\n",
      "                     'https://www.liepin.com/company/864375/',\n",
      "                     'https://www.liepin.com/company/4787049/',\n",
      "                     'https://www.liepin.com/company/9837329/',\n",
      "                     'https://www.liepin.com/company/12177731/',\n",
      "                     'https://www.liepin.com/company/10223915/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/856012/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/2589241/',\n",
      "                     'https://www.liepin.com/company/10174653/',\n",
      "                     'https://www.liepin.com/company/12177731/',\n",
      "                     'https://www.liepin.com/company/12177731/',\n",
      "                     'https://www.liepin.com/company/12177731/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/9261863/',\n",
      "                     'https://www.liepin.com/company/8582879/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/8523622/'],\n",
      " 'job_url': ['https://www.liepin.com/job/1928587613.shtml',\n",
      "             'https://www.liepin.com/job/1929562157.shtml',\n",
      "             'https://www.liepin.com/job/1929604271.shtml',\n",
      "             'https://www.liepin.com/job/1930047667.shtml',\n",
      "             'https://www.liepin.com/job/1930046827.shtml',\n",
      "             'https://www.liepin.com/job/1930046553.shtml',\n",
      "             'https://www.liepin.com/job/1930046245.shtml',\n",
      "             'https://www.liepin.com/job/1930044643.shtml',\n",
      "             'https://www.liepin.com/job/1930043139.shtml',\n",
      "             'https://www.liepin.com/job/1930041917.shtml',\n",
      "             'https://www.liepin.com/job/1930041417.shtml',\n",
      "             'https://www.liepin.com/job/1929907517.shtml',\n",
      "             'https://www.liepin.com/job/1929748725.shtml',\n",
      "             'https://www.liepin.com/job/1929387059.shtml',\n",
      "             'https://www.liepin.com/job/1929387013.shtml',\n",
      "             'https://www.liepin.com/job/1929384777.shtml',\n",
      "             'https://www.liepin.com/job/1929234127.shtml',\n",
      "             'https://www.liepin.com/job/1929234013.shtml',\n",
      "             'https://www.liepin.com/job/1927699479.shtml',\n",
      "             'https://www.liepin.com/job/1927485669.shtml',\n",
      "             'https://www.liepin.com/job/1927454335.shtml',\n",
      "             'https://www.liepin.com/job/1927362129.shtml',\n",
      "             'https://www.liepin.com/job/1927359845.shtml',\n",
      "             'https://www.liepin.com/job/1926937331.shtml',\n",
      "             'https://www.liepin.com/job/1926578663.shtml',\n",
      "             'https://www.liepin.com/job/1926578657.shtml',\n",
      "             'https://www.liepin.com/job/1924887755.shtml',\n",
      "             'https://www.liepin.com/job/1923795261.shtml',\n",
      "             '/a/21285517.shtml',\n",
      "             '/a/21261697.shtml',\n",
      "             '/a/21245609.shtml',\n",
      "             '/a/21195255.shtml',\n",
      "             '/a/21152423.shtml',\n",
      "             'https://www.liepin.com/job/1921151123.shtml',\n",
      "             '/a/21123437.shtml',\n",
      "             '/a/21113161.shtml',\n",
      "             '/a/21089531.shtml',\n",
      "             '/a/21079175.shtml',\n",
      "             '/a/21076071.shtml',\n",
      "             '/a/21070169.shtml'],\n",
      " 'job_xinshui': ['8-12k·12薪',\n",
      "                 '10-20k·12薪',\n",
      "                 '25-35k·12薪',\n",
      "                 '4-8k·12薪',\n",
      "                 '8-13k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '14-25k·12薪',\n",
      "                 '10-20k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '50-80k·12薪',\n",
      "                 '10-20k·12薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '18-25k·12薪',\n",
      "                 '15-22k·14薪',\n",
      "                 '18-29k·15薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '12-20k·13薪',\n",
      "                 '20-28k·14薪',\n",
      "                 '10-20k·14薪',\n",
      "                 '15-35k·12薪',\n",
      "                 '20-35k·13薪',\n",
      "                 '20-35k·13薪',\n",
      "                 '20-35k·13薪',\n",
      "                 '面议',\n",
      "                 '23-30k·12薪',\n",
      "                 '20-28k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '面议',\n",
      "                 '15-30k·12薪',\n",
      "                 '25-50k·13薪',\n",
      "                 '35-65k·16薪',\n",
      "                 '50-60k·12薪',\n",
      "                 '15-30k·14薪',\n",
      "                 '20-25k·12薪',\n",
      "                 '35-50k·15薪',\n",
      "                 '30-40k·12薪',\n",
      "                 '30-50k·12薪',\n",
      "                 '35-50k·16薪',\n",
      "                 '35-50k·16薪',\n",
      "                 '28-36k·12薪'],\n",
      " 'job_zhicheng': ['电商产品经理（产品开发）',\n",
      "                  '车联网产品经理',\n",
      "                  '产品经理（财务方向）',\n",
      "                  '用户产品经理（用户增长）',\n",
      "                  '移动互联网产品经理',\n",
      "                  '旅游产品经理',\n",
      "                  '互动营销产品经理',\n",
      "                  '产品外观设计经理',\n",
      "                  '产品经理/产品经理助理',\n",
      "                  '流量产品经理(003860)',\n",
      "                  '智能产品-销售经理',\n",
      "                  '产品经理（跨境电商）',\n",
      "                  '产品经理',\n",
      "                  '硬件终端产品经理',\n",
      "                  '产品经理（硬件、平台）',\n",
      "                  '毫米波雷达产品经理',\n",
      "                  'Tracker产品经理',\n",
      "                  'DVR高级产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（跨境电商）',\n",
      "                  '亚马逊产品经理',\n",
      "                  '22989-CDN产品运营经理',\n",
      "                  '高级产品经理',\n",
      "                  '平台产品经理',\n",
      "                  '国际小包 产品经理',\n",
      "                  'CS-腾讯公益产品运营经理（深圳）',\n",
      "                  '产品总监/经理',\n",
      "                  '产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '金融产品经理（总监）',\n",
      "                  '智能硬件产品经理',\n",
      "                  '前装产品经理',\n",
      "                  '资深产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '产品经理/总监',\n",
      "                  '产品策划经理-互通',\n",
      "                  '产品经理OA',\n",
      "                  '互联网产品经理'],\n",
      " 'liepin_jingyan': ['经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '5-10年'],\n",
      " 'liepin_xueli': ['大专及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '学历不限']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:40 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=4>\n",
      "\n",
      "None\n",
      "2020-07-19 14:38:42 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=3>\n",
      "\n",
      "None\n",
      "2020-07-19 14:38:43 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['某大型互联网上市公司',\n",
      "                      '广东某大型上市公司',\n",
      "                      '500强上市集团',\n",
      "                      '500强上市集团',\n",
      "                      '某综合性集团公司',\n",
      "                      '某知名 保险',\n",
      "                      '某科技公司',\n",
      "                      '国内知名制造企业',\n",
      "                      '某知名互联网公司',\n",
      "                      '广东某电器有限公司',\n",
      "                      '深圳某科技有限公司',\n",
      "                      '深圳YB科技有限公司',\n",
      "                      '某知名地产集团',\n",
      "                      '深圳某知名照明企业',\n",
      "                      '深圳某科技公司',\n",
      "                      '某公司',\n",
      "                      '深圳某科技有限公司',\n",
      "                      '某知名世界500强公司',\n",
      "                      '某知名互联网公司',\n",
      "                      '国内知名快消集团',\n",
      "                      '字节跳动',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '某知名互联网公司',\n",
      "                      '知名互联网上市公司',\n",
      "                      '深圳知名企业',\n",
      "                      '某互联网公司',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '全球知名半导体公司',\n",
      "                      '广东某主机厂',\n",
      "                      '某工业控制计算机公司',\n",
      "                      '某工业控制计算机公司',\n",
      "                      '字节跳动',\n",
      "                      '某知名互联网公司',\n",
      "                      '某知名电子商务公司',\n",
      "                      '亲子教育、早教行业头部平台',\n",
      "                      'JR',\n",
      "                      '字节跳动',\n",
      "                      '广州知名某单品牌连锁化妆品公司',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '某知名互联网公司'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/7863078/'],\n",
      " 'job_url': ['/a/21008189.shtml',\n",
      "             '/a/21007163.shtml',\n",
      "             '/a/20993557.shtml',\n",
      "             '/a/20979495.shtml',\n",
      "             '/a/20971663.shtml',\n",
      "             '/a/20962805.shtml',\n",
      "             '/a/20949965.shtml',\n",
      "             '/a/20948559.shtml',\n",
      "             '/a/20947453.shtml',\n",
      "             '/a/20945193.shtml',\n",
      "             '/a/20929091.shtml',\n",
      "             '/a/20928823.shtml',\n",
      "             '/a/20925753.shtml',\n",
      "             '/a/20925641.shtml',\n",
      "             '/a/20918127.shtml',\n",
      "             '/a/20856995.shtml',\n",
      "             '/a/20806451.shtml',\n",
      "             '/a/20768299.shtml',\n",
      "             '/a/20768145.shtml',\n",
      "             '/a/20763297.shtml',\n",
      "             'https://www.liepin.com/job/1920746237.shtml',\n",
      "             '/a/20729241.shtml',\n",
      "             '/a/20725283.shtml',\n",
      "             '/a/20718089.shtml',\n",
      "             '/a/20714741.shtml',\n",
      "             '/a/20639397.shtml',\n",
      "             '/a/20628923.shtml',\n",
      "             '/a/20541355.shtml',\n",
      "             '/a/20537961.shtml',\n",
      "             '/a/20531481.shtml',\n",
      "             '/a/20531449.shtml',\n",
      "             'https://www.liepin.com/job/1920521505.shtml',\n",
      "             '/a/20516603.shtml',\n",
      "             '/a/20472117.shtml',\n",
      "             '/a/20467735.shtml',\n",
      "             '/a/20421813.shtml',\n",
      "             'https://www.liepin.com/job/1920349503.shtml',\n",
      "             '/a/20341517.shtml',\n",
      "             '/a/20330925.shtml',\n",
      "             '/a/20193639.shtml'],\n",
      " 'job_xinshui': ['18-35k·15薪',\n",
      "                 '25-35k·15薪',\n",
      "                 '18-36k·16薪',\n",
      "                 '16-32k·16薪',\n",
      "                 '100-180k·13薪',\n",
      "                 '33-58k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '18-25k·13薪',\n",
      "                 '25-50k·13薪',\n",
      "                 '15-25k·16薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '20-30k·12薪',\n",
      "                 '28-35k·12薪',\n",
      "                 '20-30k·12薪',\n",
      "                 '20-32k·12薪',\n",
      "                 '30-50k·16薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '20-40k·13薪',\n",
      "                 '15-25k·13薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '面议',\n",
      "                 '30-50k·16薪',\n",
      "                 '20-40k·13薪',\n",
      "                 '30-45k·16薪',\n",
      "                 '15-25k·14薪',\n",
      "                 '15-30k·13薪',\n",
      "                 '15-20k·15薪',\n",
      "                 '35-50k·14薪',\n",
      "                 '20-40k·15薪',\n",
      "                 '26-37k·14薪',\n",
      "                 '26-37k·14薪',\n",
      "                 '面议',\n",
      "                 '20-35k·13薪',\n",
      "                 '18-25k·13薪',\n",
      "                 '20-35k·13薪',\n",
      "                 '20-40k·16薪',\n",
      "                 '面议',\n",
      "                 '15-25k·12薪',\n",
      "                 '15-30k·16薪',\n",
      "                 '22-30k·13薪'],\n",
      " 'job_zhicheng': ['产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（招聘系统）#214750',\n",
      "                  '产品经理（综管系统）#220484',\n",
      "                  '用户增长高级产品经理',\n",
      "                  '产品助理经理',\n",
      "                  '人工智能产品经理',\n",
      "                  '销售经理（电脑周边产品 电竞类）',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '人工智能产品经理',\n",
      "                  '人工智能产品经理',\n",
      "                  '商业会员产品经理',\n",
      "                  '产品经理',\n",
      "                  '人工智能产品经理',\n",
      "                  '（音频产品）高级产品经理',\n",
      "                  '产品经理',\n",
      "                  'ai产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '游戏安全产品经理',\n",
      "                  '信用卡产品经理',\n",
      "                  '产品经理',\n",
      "                  '资深短视频工具产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '芯片产品经理',\n",
      "                  '音乐生态产品经理',\n",
      "                  '售前产品经理',\n",
      "                  '售前产品经理',\n",
      "                  '轻颜增长产品经理',\n",
      "                  '产品经理',\n",
      "                  '网站策划/产品经理',\n",
      "                  '图书版权产品经理',\n",
      "                  '数据产品经理',\n",
      "                  '视频工具产品经理',\n",
      "                  '产品开发经理',\n",
      "                  'HR技术产品经理',\n",
      "                  '产品经理'],\n",
      " 'liepin_jingyan': ['经验不限',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年'],\n",
      " 'liepin_xueli': ['大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '学历不限',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '本科及以上']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:44 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['字节跳动',\n",
      "                      '字节跳动',\n",
      "                      '字节跳动',\n",
      "                      '半岛医疗',\n",
      "                      '国内某快速增长跨境电商公司',\n",
      "                      '帕拓逊',\n",
      "                      '某医疗用品股份有限公司',\n",
      "                      '帕拓逊',\n",
      "                      '某互联网公司',\n",
      "                      '福建某大型照明公司',\n",
      "                      '知名独角兽科技公司',\n",
      "                      '北美汽车消费品排名前三',\n",
      "                      '某互联网保险公司',\n",
      "                      '某互联网公司',\n",
      "                      '国内TMD某知名互联网企业',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '某知名互联网科技公司',\n",
      "                      '知名券商',\n",
      "                      '深圳xxx科技有限公司',\n",
      "                      '国内某知名互联网公司',\n",
      "                      '深圳某AI自动化招聘系统',\n",
      "                      '某上市通信集团',\n",
      "                      '深圳上市跨境电商公司',\n",
      "                      '深圳知名企业',\n",
      "                      '深圳市某公司（保险科技领域）',\n",
      "                      '某保险集团',\n",
      "                      'BL',\n",
      "                      '高端离子源技术研发及整机装备制造公司 ',\n",
      "                      '某互联网平台企业',\n",
      "                      '深圳市某科技有限公司',\n",
      "                      '某综合性集团公司',\n",
      "                      '某科技有限公司',\n",
      "                      '娱乐直播平台企业',\n",
      "                      '深圳知名互联网企业',\n",
      "                      '某知名互联网公司',\n",
      "                      '深圳某科技有限公司',\n",
      "                      '国内知名互联网公司',\n",
      "                      '某光电（深圳）有限公司',\n",
      "                      '深圳S公司',\n",
      "                      '500强上市集团'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/8849807/'],\n",
      " 'job_url': ['https://www.liepin.com/job/1921763113.shtml',\n",
      "             'https://www.liepin.com/job/1921633251.shtml',\n",
      "             'https://www.liepin.com/job/1921393731.shtml',\n",
      "             'https://www.liepin.com/job/1921388183.shtml',\n",
      "             '/a/21329215.shtml',\n",
      "             '/a/21328737.shtml',\n",
      "             '/a/21328725.shtml',\n",
      "             '/a/21328715.shtml',\n",
      "             '/a/21321649.shtml',\n",
      "             '/a/21320655.shtml',\n",
      "             '/a/21314163.shtml',\n",
      "             '/a/21301641.shtml',\n",
      "             '/a/21299045.shtml',\n",
      "             '/a/21276011.shtml',\n",
      "             '/a/21269891.shtml',\n",
      "             '/a/21261705.shtml',\n",
      "             '/a/21254217.shtml',\n",
      "             '/a/21228717.shtml',\n",
      "             '/a/21211387.shtml',\n",
      "             '/a/21195639.shtml',\n",
      "             '/a/21176601.shtml',\n",
      "             '/a/21171915.shtml',\n",
      "             '/a/21171155.shtml',\n",
      "             '/a/21171057.shtml',\n",
      "             '/a/21152073.shtml',\n",
      "             '/a/21132973.shtml',\n",
      "             '/a/21097629.shtml',\n",
      "             '/a/21095191.shtml',\n",
      "             '/a/21094955.shtml',\n",
      "             '/a/21091543.shtml',\n",
      "             '/a/21091251.shtml',\n",
      "             '/a/21071349.shtml',\n",
      "             '/a/21063523.shtml',\n",
      "             '/a/21063315.shtml',\n",
      "             '/a/21052141.shtml',\n",
      "             '/a/21051025.shtml',\n",
      "             '/a/21034441.shtml',\n",
      "             '/a/21021227.shtml',\n",
      "             '/a/21016093.shtml',\n",
      "             '/a/21014313.shtml'],\n",
      " 'job_xinshui': ['面议',\n",
      "                 '面议',\n",
      "                 '面议',\n",
      "                 '10-20k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '20-35k·14薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '20-35k·14薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '20-30k·13薪',\n",
      "                 '30-60k·15薪',\n",
      "                 '15-25k·13薪',\n",
      "                 '20-35k·15薪',\n",
      "                 '25-50k·13薪',\n",
      "                 '25-50k·16薪',\n",
      "                 '20-30k·16薪',\n",
      "                 '4-5k·12薪',\n",
      "                 '40-60k·18薪',\n",
      "                 '20-35k·12薪',\n",
      "                 '60-90k·12薪',\n",
      "                 '25-50k·12薪',\n",
      "                 '20-35k·15薪',\n",
      "                 '25-35k·15薪',\n",
      "                 '15-25k·14薪',\n",
      "                 '20-40k·13薪',\n",
      "                 '20-40k·14薪',\n",
      "                 '25-30k·13薪',\n",
      "                 '16-20k·13薪',\n",
      "                 '83-108k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '60-90k·16薪',\n",
      "                 '20-40k·13薪',\n",
      "                 '15-25k·14薪',\n",
      "                 '25-35k·13薪',\n",
      "                 '25-50k·13薪',\n",
      "                 '15-25k·13薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '25-45k·14薪',\n",
      "                 '18-28k·16薪'],\n",
      " 'job_zhicheng': ['影像产品经理-Faceu',\n",
      "                  '增长效率产品经理',\n",
      "                  '高级产品经理-内容社区方向',\n",
      "                  '产品经理/皮科医生',\n",
      "                  '产品经理',\n",
      "                  '产品经理（财务）',\n",
      "                  '成人失禁护理产品经理',\n",
      "                  '产品经理（吸尘器）',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '大数据产品经理',\n",
      "                  'EBay/亚马逊产品经理',\n",
      "                  '资深产品经理',\n",
      "                  '产品经理',\n",
      "                  '高级海外产品运营经理',\n",
      "                  '数据产品经理',\n",
      "                  '产品企划经理',\n",
      "                  '结构金融产品经理',\n",
      "                  '运营产品经理S2B2C',\n",
      "                  '电商产品总经理/CTO',\n",
      "                  '产品总监/经理',\n",
      "                  '产品经理（流程管理）#222409',\n",
      "                  '产品经理（吸尘器）',\n",
      "                  '产品经理',\n",
      "                  '人工智能产品经理',\n",
      "                  'ai产品经理',\n",
      "                  '产品经理(智能家电方向)',\n",
      "                  '销售部产品经理（真空镀膜设备 离子源）售前技术支持',\n",
      "                  '产品总监/经理',\n",
      "                  '产品经理',\n",
      "                  '推荐策略产品经理-150805',\n",
      "                  '产品经理',\n",
      "                  '直播产品经理',\n",
      "                  '高级产品经理（TMS）',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '前端产品经理',\n",
      "                  '产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '产品经理（智慧薪酬）#222137'],\n",
      " 'liepin_jingyan': ['3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '经验不限'],\n",
      " 'liepin_xueli': ['本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '硕士及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '学历不限',\n",
      "                  '学历不限',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:46 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['某知名互联网企业',\n",
      "                      '某知名地产集团',\n",
      "                      '某知名地产集团',\n",
      "                      '某知名地产集团',\n",
      "                      '某大型互联网上市公司',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '某高速发展快消集团',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '某知名互联网公司',\n",
      "                      '蓝禾科技集团公司',\n",
      "                      '蓝禾科技集团公司',\n",
      "                      '蓝禾科技集团公司',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '某知名互联网公司',\n",
      "                      'SKSKJ',\n",
      "                      '某知名互联网公司',\n",
      "                      '知名金融公司科技板块',\n",
      "                      '某大型互联网公司',\n",
      "                      '国内知名护肤品单品牌连锁店',\n",
      "                      '国企保险公司（TOP前5）',\n",
      "                      '某股份有限公司',\n",
      "                      '深圳上市跨境电商公司',\n",
      "                      '字节跳动',\n",
      "                      '字节跳动',\n",
      "                      '华傲数据',\n",
      "                      '周大生',\n",
      "                      '深圳市中明科技股份有限公司',\n",
      "                      '字节跳动',\n",
      "                      '湖北世纪联合创新科技有限公司',\n",
      "                      '湖北世纪联合创新科技有限公司',\n",
      "                      '某大型互联网公司',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '地上铁租车(深圳)有限公司',\n",
      "                      '深圳市纳音科技有限公司',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '亚能生物',\n",
      "                      '湖北世纪联合创新科技有限公司',\n",
      "                      '华傲数据',\n",
      "                      '华傲数据',\n",
      "                      '银雁科技服务集团股份有限公司'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/6825371/',\n",
      "                     'https://www.liepin.com/company/8029807/',\n",
      "                     'https://www.liepin.com/company/9589477/',\n",
      "                     'https://www.liepin.com/company/7863078/',\n",
      "                     'https://www.liepin.com/company/9424515/',\n",
      "                     'https://www.liepin.com/company/9424515/',\n",
      "                     'https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/8889691/',\n",
      "                     'https://www.liepin.com/company/9547076/',\n",
      "                     'https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/8198433/',\n",
      "                     'https://www.liepin.com/company/9424515/',\n",
      "                     'https://www.liepin.com/company/6825371/',\n",
      "                     'https://www.liepin.com/company/6825371/',\n",
      "                     'https://www.liepin.com/company/795969/'],\n",
      " 'job_url': ['/a/20193389.shtml',\n",
      "             '/a/20185747.shtml',\n",
      "             '/a/20185689.shtml',\n",
      "             '/a/20185665.shtml',\n",
      "             '/a/20184833.shtml',\n",
      "             '/a/20169501.shtml',\n",
      "             '/a/20167507.shtml',\n",
      "             'https://www.liepin.com/job/1920124969.shtml',\n",
      "             '/a/20101305.shtml',\n",
      "             '/a/20085799.shtml',\n",
      "             '/a/20082531.shtml',\n",
      "             '/a/20082457.shtml',\n",
      "             'https://www.liepin.com/job/1920080725.shtml',\n",
      "             '/a/20077635.shtml',\n",
      "             '/a/19982105.shtml',\n",
      "             '/a/19964981.shtml',\n",
      "             '/a/19958087.shtml',\n",
      "             '/a/19947443.shtml',\n",
      "             '/a/19935287.shtml',\n",
      "             '/a/19916373.shtml',\n",
      "             '/a/19765907.shtml',\n",
      "             '/a/19138015.shtml',\n",
      "             'https://www.liepin.com/job/1918820237.shtml',\n",
      "             'https://www.liepin.com/job/1918699515.shtml',\n",
      "             'https://www.liepin.com/job/1918647279.shtml',\n",
      "             'https://www.liepin.com/job/1918497073.shtml',\n",
      "             'https://www.liepin.com/job/1918362859.shtml',\n",
      "             'https://www.liepin.com/job/1918078047.shtml',\n",
      "             'https://www.liepin.com/job/1917748543.shtml',\n",
      "             'https://www.liepin.com/job/1917746889.shtml',\n",
      "             '/a/17494173.shtml',\n",
      "             'https://www.liepin.com/job/1917412635.shtml',\n",
      "             'https://www.liepin.com/job/1917316961.shtml',\n",
      "             'https://www.liepin.com/job/1916591481.shtml',\n",
      "             'https://www.liepin.com/job/1916163999.shtml',\n",
      "             'https://www.liepin.com/job/1915387000.shtml',\n",
      "             'https://www.liepin.com/job/1915315449.shtml',\n",
      "             'https://www.liepin.com/job/1914439557.shtml',\n",
      "             'https://www.liepin.com/job/1914439356.shtml',\n",
      "             'https://www.liepin.com/job/1913417448.shtml'],\n",
      " 'job_xinshui': ['25-35k·16薪',\n",
      "                 '30-50k·12薪',\n",
      "                 '28-50k·12薪',\n",
      "                 '28-40k·12薪',\n",
      "                 '20-40k·15薪',\n",
      "                 '20-35k·16薪',\n",
      "                 '15-30k·13薪',\n",
      "                 '面议',\n",
      "                 '20-30k·13薪',\n",
      "                 '20-40k·12薪',\n",
      "                 '25-40k·16薪',\n",
      "                 '20-40k·16薪',\n",
      "                 '15-18k·12薪',\n",
      "                 '25-40k·12薪',\n",
      "                 '20-30k·13薪',\n",
      "                 '15-25k·13薪',\n",
      "                 '30-50k·16薪',\n",
      "                 '15-30k·14薪',\n",
      "                 '18-22k·12薪',\n",
      "                 '15-25k·14薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '20-35k·13薪',\n",
      "                 '面议',\n",
      "                 '面议',\n",
      "                 '20-30k·12薪',\n",
      "                 '8-16k·12薪',\n",
      "                 '6-12k·12薪',\n",
      "                 '面议',\n",
      "                 '12-25k·12薪',\n",
      "                 '12-25k·13薪',\n",
      "                 '30-50k·12薪',\n",
      "                 '面议',\n",
      "                 '15-25k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '面议',\n",
      "                 '8-12k·14薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '18-30k·12薪',\n",
      "                 '18-30k·12薪',\n",
      "                 '10-15k·12薪'],\n",
      " 'job_zhicheng': ['高级产品经理',\n",
      "                  '数据中台产品经理',\n",
      "                  '超级APP产品经理',\n",
      "                  '中台产品经理',\n",
      "                  '前端产品经理',\n",
      "                  'APP产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（账户）(J11770)',\n",
      "                  '高级产品经理',\n",
      "                  '硬件产品经理',\n",
      "                  '硬件产品经理',\n",
      "                  '硬件产品经理',\n",
      "                  '产品经理（创新产品）(J12588)',\n",
      "                  '产品经理',\n",
      "                  '产品经理（小家电）',\n",
      "                  '产品经理',\n",
      "                  '高级数据产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品开发经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（厨电，小家电类）',\n",
      "                  '用户中心产品经理',\n",
      "                  '数据产品经理',\n",
      "                  '产品经理（政务大数据/智慧城市）',\n",
      "                  '珠宝产品经理-互联天下',\n",
      "                  'ESD教务主任/ESD工程师/ESD产品经理',\n",
      "                  '风控产品经理-数据方向',\n",
      "                  '电子白板APP产品经理',\n",
      "                  '电白板产品经理',\n",
      "                  '高级产品经理',\n",
      "                  '产品经理（信贷）(J10662)',\n",
      "                  '产品经理',\n",
      "                  '产品经理',\n",
      "                  '产品经理（账户）J11770',\n",
      "                  '产品经理',\n",
      "                  '商用显示产品经理',\n",
      "                  '产品经理（数据分析、数据可视化方向）',\n",
      "                  '产品经理（数据治理方向）',\n",
      "                  '产品经理（信贷）'],\n",
      " 'liepin_jingyan': ['1-3年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '1-3年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '10年以上',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '10年以上',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年'],\n",
      " 'liepin_xueli': ['学历不限',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '学历不限',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:48 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.liepin.com/zhaopin/?dqs=050090&searchType=1&sortFlag=15&d_headId=129e5737ebc361c490c7d454acdaac19&d_ckId=cac543a820da194c05ab6546987bbea4&d_sfrom=search_unknown&d_curPage=0&d_pageSize=40&siTag=1B2M2Y8AsgTpgAmY7PhCfg~-nQsjvAMdjst7vnBI-6VZQ&key=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&curPage=9>\n",
      "\n",
      "None\n",
      "2020-07-19 14:38:49 [scrapy.core.scraper] ERROR: Error processing {'job_company_name': ['银雁科技服务集团股份有限公司',\n",
      "                      '银雁科技服务集团股份有限公司',\n",
      "                      '周大生',\n",
      "                      '鹏元征信',\n",
      "                      '深圳某电源研发公司',\n",
      "                      '某人力资源公司',\n",
      "                      '深圳欧陆通电子股份有限公司',\n",
      "                      '知名量化对冲基金公司',\n",
      "                      '顺丰科技',\n",
      "                      '平安科技',\n",
      "                      '惠州市三生滋本投资管理有限公司',\n",
      "                      '深圳市即构科技有限公司',\n",
      "                      '华润集团信息管理部',\n",
      "                      '奥尼电子',\n",
      "                      '腾讯',\n",
      "                      '深圳市法本信息技术股份有限公司',\n",
      "                      '深圳市社群独角兽科技有限公司',\n",
      "                      '深圳市宏铨煋科技有限公司',\n",
      "                      '深圳市社群独角兽科技有限公司',\n",
      "                      '深圳市易思博酷客科技有限公司',\n",
      "                      '依米康智能工程有限公司',\n",
      "                      '深圳爱问科技股份有限公司',\n",
      "                      '深圳市天行云供应链有限公司',\n",
      "                      '珍爱网',\n",
      "                      'vivo',\n",
      "                      'vivo',\n",
      "                      '匯智國際旅遊發展有限公司',\n",
      "                      '傲基科技股份有限公司',\n",
      "                      '傲基科技股份有限公司',\n",
      "                      '深圳卓正医疗投资咨询有限公司',\n",
      "                      '深圳点猫科技有限公司',\n",
      "                      '兔展-RabbitPre',\n",
      "                      '深圳市几素科技有限公司',\n",
      "                      '智云',\n",
      "                      '深圳市社群独角兽科技有限公司',\n",
      "                      '润泊城(深圳)商务咨询有限公司',\n",
      "                      '房多多',\n",
      "                      '优学派智慧教育',\n",
      "                      '梦享',\n",
      "                      '炬星科技(深圳)有限公司'],\n",
      " 'job_company_url': ['https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/795969/',\n",
      "                     'https://www.liepin.com/company/8029807/',\n",
      "                     'https://www.liepin.com/company/8088241/',\n",
      "                     'https://www.liepin.com/company/9692222/',\n",
      "                     'https://www.liepin.com/company/8353564/',\n",
      "                     'https://www.liepin.com/company/1086536/',\n",
      "                     'https://www.liepin.com/company/9427587/',\n",
      "                     'https://www.liepin.com/company/9725383/',\n",
      "                     'https://www.liepin.com/company/9249205/',\n",
      "                     'https://www.liepin.com/company/9143628/',\n",
      "                     'https://www.liepin.com/company/7983148/',\n",
      "                     'https://www.liepin.com/company/7604446/',\n",
      "                     'https://www.liepin.com/company/12243277/',\n",
      "                     'https://www.liepin.com/company/12313595/',\n",
      "                     'https://www.liepin.com/company/12243277/',\n",
      "                     'https://www.liepin.com/company/9641148/',\n",
      "                     'https://www.liepin.com/company/10176931/',\n",
      "                     'https://www.liepin.com/company/9323021/',\n",
      "                     'https://www.liepin.com/company/8802637/',\n",
      "                     'https://www.liepin.com/company/937933/',\n",
      "                     'https://www.liepin.com/company/4787049/',\n",
      "                     'https://www.liepin.com/company/4787049/',\n",
      "                     'https://www.liepin.com/company/10094557/',\n",
      "                     'https://www.liepin.com/company/8690691/',\n",
      "                     'https://www.liepin.com/company/8690691/',\n",
      "                     'https://www.liepin.com/company/8302264/',\n",
      "                     'https://www.liepin.com/company/8632721/',\n",
      "                     'https://www.liepin.com/company/8161746/',\n",
      "                     'https://www.liepin.com/company/9589154/',\n",
      "                     'https://www.liepin.com/company/8891221/',\n",
      "                     'https://www.liepin.com/company/12243277/',\n",
      "                     'https://www.liepin.com/company/12281939/',\n",
      "                     'https://www.liepin.com/company/7871137/',\n",
      "                     'https://www.liepin.com/company/10244357/',\n",
      "                     'https://www.liepin.com/company/8336246/',\n",
      "                     'https://www.liepin.com/company/9657728/'],\n",
      " 'job_url': ['https://www.liepin.com/job/1912200895.shtml',\n",
      "             'https://www.liepin.com/job/1912199921.shtml',\n",
      "             'https://www.liepin.com/job/198625981.shtml',\n",
      "             'https://www.liepin.com/job/196746573.shtml',\n",
      "             '/a/21169061.shtml',\n",
      "             '/a/20218975.shtml',\n",
      "             'https://www.liepin.com/job/1923602953.shtml',\n",
      "             '/a/21093489.shtml',\n",
      "             'https://www.liepin.com/job/1930023287.shtml',\n",
      "             'https://www.liepin.com/job/1930023029.shtml',\n",
      "             'https://www.liepin.com/job/1930022447.shtml',\n",
      "             'https://www.liepin.com/job/1930022257.shtml',\n",
      "             'https://www.liepin.com/job/1930019873.shtml',\n",
      "             'https://www.liepin.com/job/1930019575.shtml',\n",
      "             'https://www.liepin.com/job/1930018213.shtml',\n",
      "             'https://www.liepin.com/job/1930017851.shtml',\n",
      "             'https://www.liepin.com/job/1930017695.shtml',\n",
      "             'https://www.liepin.com/job/1930016953.shtml',\n",
      "             'https://www.liepin.com/job/1930016801.shtml',\n",
      "             'https://www.liepin.com/job/1930015561.shtml',\n",
      "             'https://www.liepin.com/job/1930013411.shtml',\n",
      "             'https://www.liepin.com/job/1930012831.shtml',\n",
      "             'https://www.liepin.com/job/1930006847.shtml',\n",
      "             'https://www.liepin.com/job/1930006493.shtml',\n",
      "             'https://www.liepin.com/job/1930005149.shtml',\n",
      "             'https://www.liepin.com/job/1930004625.shtml',\n",
      "             'https://www.liepin.com/job/1930003929.shtml',\n",
      "             'https://www.liepin.com/job/1930003893.shtml',\n",
      "             'https://www.liepin.com/job/1930003811.shtml',\n",
      "             'https://www.liepin.com/job/1930002743.shtml',\n",
      "             'https://www.liepin.com/job/1930002061.shtml',\n",
      "             'https://www.liepin.com/job/1929997131.shtml',\n",
      "             'https://www.liepin.com/job/1929995389.shtml',\n",
      "             'https://www.liepin.com/job/1929995081.shtml',\n",
      "             'https://www.liepin.com/job/1929986497.shtml',\n",
      "             'https://www.liepin.com/job/1929983127.shtml',\n",
      "             'https://www.liepin.com/job/1929980715.shtml',\n",
      "             'https://www.liepin.com/job/1929974989.shtml',\n",
      "             'https://www.liepin.com/job/1929952205.shtml',\n",
      "             'https://www.liepin.com/job/1929949979.shtml'],\n",
      " 'job_xinshui': ['6-10k·12薪',\n",
      "                 '6-9k·12薪',\n",
      "                 '10-15k·15薪',\n",
      "                 '18-24k·12薪',\n",
      "                 '17-20k·12薪',\n",
      "                 '14-20k·12薪',\n",
      "                 '8-10k·13薪',\n",
      "                 '40-60k·20薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '面议',\n",
      "                 '15-20k·12薪',\n",
      "                 '15-25k·15薪',\n",
      "                 '9-15k·12薪',\n",
      "                 '15-30k·12薪',\n",
      "                 '面议',\n",
      "                 '12-18k·12薪',\n",
      "                 '20-30k·12薪',\n",
      "                 '8-12k·12薪',\n",
      "                 '20-40k·13薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '20-25k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '25-35k·12薪',\n",
      "                 '25-50k·12薪',\n",
      "                 '25-50k·12薪',\n",
      "                 '10-15k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '20-30k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '16-28k·12薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '15-25k·12薪',\n",
      "                 '15-40k·12薪',\n",
      "                 '12-20k·13薪',\n",
      "                 '15-20k·12薪',\n",
      "                 '25-35k·13薪',\n",
      "                 '15-22k·12薪',\n",
      "                 '12-18k·12薪'],\n",
      " 'job_zhicheng': ['产品经理（收单项目）',\n",
      "                  '产品经理（网点管理）',\n",
      "                  '产品分析经理',\n",
      "                  '高级产品经理',\n",
      "                  '项目经理pm',\n",
      "                  'pm项目经理',\n",
      "                  'PM（市场部）',\n",
      "                  '量化PM/基金经理/投资经理',\n",
      "                  'OO1993-产品经理',\n",
      "                  '1131EA-智能安防产品经理',\n",
      "                  '资深产品经理',\n",
      "                  '在线教育产品经理',\n",
      "                  '华润网络产品经理',\n",
      "                  '硬件产品经理（IOT）',\n",
      "                  '18432-产品运营经理 (通讯业务)',\n",
      "                  '互联网产品经理',\n",
      "                  '短视频商业产品经理-广告体验方向',\n",
      "                  '产品经理',\n",
      "                  '用户产品经理',\n",
      "                  '互联网产品经理',\n",
      "                  '机器人产品经理',\n",
      "                  '后台产品经理',\n",
      "                  '产品经理',\n",
      "                  '高级产品经理（内容产品）',\n",
      "                  '搜索广告产品经理(007996)',\n",
      "                  '海外商业产品经理(007995)',\n",
      "                  '国内酒店产品经理（深圳）',\n",
      "                  '产品开发经理',\n",
      "                  '产品开发经理（工具）',\n",
      "                  '高级产品经理（互联网医疗）',\n",
      "                  '创作工具产品经理',\n",
      "                  '交互型产品经理',\n",
      "                  '产品营销策划经理',\n",
      "                  '产品经理（智能硬件）',\n",
      "                  '产品经理（自我诊疗方向）',\n",
      "                  '金融产品经理',\n",
      "                  '法务经理（产品方向）(J11063)',\n",
      "                  '资深产品经理（部门经理职级）',\n",
      "                  '广告产品经理',\n",
      "                  '助理产品经理（偏软件）'],\n",
      " 'liepin_jingyan': ['1-3年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '1-3年',\n",
      "                    '1-3年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '3-5年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '1-3年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '经验不限',\n",
      "                    '5-10年',\n",
      "                    '3-5年',\n",
      "                    '1-3年'],\n",
      " 'liepin_xueli': ['统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '硕士及以上',\n",
      "                  '本科及以上',\n",
      "                  '学历不限',\n",
      "                  '大专及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '大专及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '本科及以上',\n",
      "                  '统招本科',\n",
      "                  '统招本科',\n",
      "                  '本科及以上',\n",
      "                  '统招本科']}\n",
      "Traceback (most recent call last):\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\twisted\\internet\\defer.py\", line 654, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"C:\\Users\\Administrator.DESKTOP-IC2NVKM\\Desktop\\20春_Web数据挖掘\\liepin\\liepin\\pipelines.py\", line 17, in process_item\n",
      "    df[\"公司链接\"]=item[\"job_company_url\"]\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 2938, in __setitem__\n",
      "    self._set_item(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3000, in _set_item\n",
      "    value = self._sanitize_column(key, value)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\frame.py\", line 3636, in _sanitize_column\n",
      "    value = sanitize_index(value, self.index, copy=False)\n",
      "  File \"F:\\qq\\python\\lib\\site-packages\\pandas\\core\\internals\\construction.py\", line 611, in sanitize_index\n",
      "    raise ValueError(\"Length of values does not match length of index\")\n",
      "ValueError: Length of values does not match length of index\n",
      "2020-07-19 14:38:49 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2020-07-19 14:38:49 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 4910,\n",
      " 'downloader/request_count': 10,\n",
      " 'downloader/request_method_count/GET': 10,\n",
      " 'downloader/response_bytes': 223551,\n",
      " 'downloader/response_count': 10,\n",
      " 'downloader/response_status_count/200': 10,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2020, 7, 19, 6, 38, 49, 583788),\n",
      " 'item_scraped_count': 3,\n",
      " 'log_count/DEBUG': 13,\n",
      " 'log_count/ERROR': 7,\n",
      " 'log_count/INFO': 9,\n",
      " 'response_received_count': 10,\n",
      " 'scheduler/dequeued': 10,\n",
      " 'scheduler/dequeued/memory': 10,\n",
      " 'scheduler/enqueued': 10,\n",
      " 'scheduler/enqueued/memory': 10,\n",
      " 'start_time': datetime.datetime(2020, 7, 19, 6, 38, 34, 404319)}\n",
      "2020-07-19 14:38:49 [scrapy.core.engine] INFO: Spider closed (finished)\n"
     ]
    }
   ],
   "source": [
    "!scrapy crawl liepinSpider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "df=pd.read_csv(\"liepin_jobs.csv\",sep=\"\\t\",encoding=\"utf-8\")#[\"liepin_final\"].str.contains('\"0')#.loc[0,\"liepin_job\"]\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(\"liepin.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[0,\"liepin_job\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
